#include "stdafx.h"
#include <spinn3r.h>
#include <memes.h>
//#include "Spinn3rToLDA.h"

void BigMain(int argc, char* argv[]);
void ParseFromSpinn3r();
void SaveForFlash(const PQuoteBs& QtBs, const TStr Pref="", const bool& TakeOnlyRoots=true, const TUrlTy& CntUrlTy=utUndef, const TIntV& IncludeCIdV=TIntV());
void PlotsForPaper();
void SaveQtDomNet(const int& CId, const PQuoteBs& QtBs, const PNGraph& PostGraph, const TStr& OutFNm);

using namespace TSnap;

int main(int argc, char* argv[]) {
  printf("QuotesApp. build: %s, %s. Start time: %s\n\n", __TIME__, __DATE__, TExeTm::GetCurTm());
  //TSysProc::SetLowPriority();
  TExeTm ExeTm;  TInt::Rnd.PutSeed(0);  Try
  //BigMain(argc, argv); return 0;
  //char *ToDo [] = {"QuotesApp", "-do:mkdataset", "-i:f.txt"};  BigMain(3, ToDo);  return 0;
  //char *ToDo [] = {"QuotesApp", "-do:linkcascades", "-i:b.txt"};  BigMain(3, ToDo);  return 0;

  //TQuoteExtractor QE(TStr("W:\\Projects\\MemeTracker\\MemesApp\\AWorkDir\\data\\fullSpinnr-*.gz"));
  //for (int item = 0; QE.Next(); item++) { }

  //////
  /*PQuoteBs QtBs = TQuoteBs::Load(TZipIn("QtBase-qt0808-0902w3mfq5\\qt08080902w3mfq5-clust.QtBs.rar"));
  //QtBs->GetQuotePostNet("W:\\Data\\MemesDataset\\quotes_200*.txt.rar")->Save(TFOut("qt08080902w3mfq5.QtPostNet"));
  PNGraph G = TNGraph::Load(TFIn("qt08080902w3mfq5.QtPostNet"));
  TIntV CIdV;  QtBs->GetCIdVByFq(CIdV, 100);
  TFltPrV NV, EV, AvgCcSzV, MxCcSzV;
  TMom NMom, EMom, AvgCcSzFrac, MxCcSzFrac;
  for (int c = 0; c < 1000; c++) {
    //SaveQtDomNet(CIdV[c], QtBs, G, TStr::Fmt("qt-%03d", c+1));
    PQtDomNet DomNet = QtBs->GetQuoteDomNet(G, CIdV[c]);
    TCnComV Dcc, Bcc;
    TSnap::GetWccs(DomNet, Dcc);  Dcc.Add(TCnCom(TIntV::GetV(0)));
    TIntV BlogV;
    for (TQtDomNet::TNodeI NI = DomNet->BegNI(); NI < DomNet->EndNI(); NI++) {
      if (NI().Val2 == utBlog) { BlogV.Add(NI.GetId()); }
    }
    PQtDomNet BlogNet = TSnap::GetSubGraph(DomNet, BlogV);
    TSnap::GetWccs(BlogNet, Bcc); Bcc.Add(TCnCom(TIntV::GetV(0)));
    if (DomNet->GetNodes() == 0) { continue; }
    if (DomNet->GetEdges() == 0) { continue; }
    NV.Add(TFltPr(DomNet->GetNodes(), BlogNet->GetNodes()));
    EV.Add(TFltPr(DomNet->GetEdges(), BlogNet->GetEdges()));
    if (BlogNet->GetNodes()!=0) {
      NMom.Add(BlogNet->GetNodes()/(double)DomNet->GetNodes()); }
    if (BlogNet->GetEdges()!=0) {
      EMom.Add(BlogNet->GetEdges()/(double)DomNet->GetEdges()); }
    AvgCcSzV.Add(TFltPr(Dcc.Len()/(double)DomNet->GetNodes(), Bcc.Len()/(double)BlogNet->GetNodes()));
    MxCcSzV.Add(TFltPr(Dcc[0].Len(), Bcc[0].Len()));

    AvgCcSzFrac.Add(Bcc.Len()/(double)Dcc.Len());
    MxCcSzFrac.Add(Bcc[0].Len()/(double)Dcc[0].Len());
  }
  NMom.Def(); EMom.Def(); AvgCcSzFrac.Def(); MxCcSzFrac.Def();
  printf("\nfraction of blog nodes: %f  %f\n", NMom.GetMean(), NMom.GetMedian());
  printf("fraction of blog edges: %f  %f\n", EMom.GetMean(), EMom.GetMedian());
  printf("blog/media number of ccs: %f  %f\n", AvgCcSzFrac.GetMean(), AvgCcSzFrac.GetMedian());
  printf("blog/media mx cc size: %f  %f\n", MxCcSzFrac.GetMean(), MxCcSzFrac.GetMedian());
  TGnuPlot::PlotValV(NV, "bmNet-Nodes", "Blog-Media network", "Number of blog+media nodes", "Number of blog nodes", gpsLog, false, gpwPoints);
  TGnuPlot::PlotValV(EV, "bmNet-Edges", "Blog-Media network", "Number of blog+media edges", "Number of blog-blog edges", gpsLog, false, gpwPoints);
  TGnuPlot::PlotValV(AvgCcSzV, "bmNet-ccSz", "Blog-Media network", "Avg. CC size in blog+media network", "Avg. CC size in blog network", gpsLog, false, gpwPoints);
  TGnuPlot::PlotValV(MxCcSzV, "bmNet-mxCcSz", "Blog-Media network", "Mx CC size in blog+media network", "Mx CC size in blog network", gpsLog, false, gpwPoints);

  /*QtBs->PlotMediaVsBlogLag(tmu30Min, true, 10000, "top1kC30min10k");
  QtBs->PlotMediaVsBlogLag(tmu1Hour, true, 10000, "top1kC01h10k");
  QtBs->PlotMediaVsBlogLag(tmu2Hour, true, 10000, "top1kC02h10k");
  QtBs->PlotMediaVsBlogLag(tmu1Min, true, 10000, "top1kC01min10k");
  QtBs->PlotMediaVsBlogLag(tmu10Min, true, 10000, "top1kC10min10k");
  QtBs->PlotMediaVsBlogLag(tmu15Min, true, 10000, "top1kC15min10k");*/
  //TStr Pref ="PeakDivMB-";
  /*QtBs->PlotBlogVsMediaFqOt(tmu1Sec, true, 1000, Pref+"1k-01sec");
  QtBs->PlotBlogVsMediaFqOt(tmu1Min, true, 1000, Pref+"1K-01min");
  QtBs->PlotBlogVsMediaFqOt(tmu15Min, true, 1000, Pref+"1K-15min");
  QtBs->PlotBlogVsMediaFqOt(tmu1Sec, true, 10000, Pref+"10K-01sec");
  QtBs->PlotBlogVsMediaFqOt(tmu1Min, true, 10000, Pref+"10K-01min");
  QtBs->PlotBlogVsMediaFqOt(tmu15Min, true, 10000, Pref+"10K-15min");*/
  //QtBs->PlotBlogVsMediaFqOt(tmu1Hour, true, 1000, Pref+"1K-1h");
  //QtBs->PlotBlogVsMediaFqOt(tmu1Hour, true, 10000, Pref+"10K-1h");

  /*TStrV FromDomains;
  for (TSsParser Ss("QtBase-qt0808-0902w3mfq5/science.txt", ssfTabSep); Ss.Next(); ) { FromDomains.Add(Ss[0]); }
  TSecTm MinTm, MaxTm; QtBs->GetMinMaxTm(MinTm, MaxTm);
  TIntV QtIdV;
  TStr Pref="scienceQ5-";
  QtBs->GetQtIdVByFq(QtIdV, 5, 100, FromDomains, true, "", 1);
  //QtBs->GetQtIdVByFq(QtIdV, 0, 5, false, "scien", "", utUndef);
  QtBs->SaveForFlash(QtIdV, Pref+"top30", tmu4Hour, 30, TSecTm(1), TSecTm(TInt::Mx-1), utUndef);
  QtBs->SaveForFlash(QtIdV, Pref+"top50", tmu4Hour, 50, TSecTm(1), TSecTm(TInt::Mx-1), utUndef);
  QtBs->SaveForFlash(QtIdV, Pref+"top100", tmu4Hour, 100, TSecTm(1), TSecTm(TInt::Mx-1), utUndef);
  /*Pref="scienceU12h-ScU-";
  //QtBs->GetQtIdVByFq(QtIdV, 0, 5, false, "", "scien", utUndef);
  QtBs->SaveForFlash(QtIdV, Pref+"top30", tmu12Hour, 30, TSecTm(1), TSecTm(TInt::Mx-1), utUndef);
  QtBs->SaveForFlash(QtIdV, Pref+"top50", tmu12Hour, 50, TSecTm(1), TSecTm(TInt::Mx-1), utUndef);
  QtBs->SaveForFlash(QtIdV, Pref+"top100", tmu12Hour, 100, TSecTm(1), TSecTm(TInt::Mx-1), utUndef);
  */

  //// save top clusters over time (for SIR fitting)
  /*PQuoteBs QtBs = TQuoteBs::Load(TZipIn("QtBase-qt0808-0902w3mfq5\\qt08080902w3mfq5-clust.QtBs.rar"));
  const TTmUnit TmUnit = tmuDay;
  const int TmUnitSecs = TTmInfo::GetTmUnitSecs(TmUnit);
  TIntV QtIdV;
  TQuote::TTmFltPrV RawFqV, FqV;
  THash <TSecTm, TFltTr> AllMedBlgH;
  QtBs->GetCIdVByFq(QtIdV, 100, "", utUndef, false); // only clusters
  //QtBs->GetQtIdVByFq(QtIdV, 3, 100, false, "", utUndef);
  printf("%d clustes with at least frequency of 100", QtIdV.Len());
  FILE *F = fopen("clust-fq-10k.txt", "wt");
  TFOut FOut("clustMB-overTm-10k.StrFltPrV");
  int NPeak1 = 0;
  for (int q = 0; q < QtIdV.Len() && NPeak1 < 10000; q++) {
    TQuote Qt; QtBs->GetMergedClustQt(QtIdV[q], Qt, false);
    //const TQuote& Qt = QtBs->GetQt(QtIdV[q]);
    const bool Peak1M = Qt.IsSinglePeak(TmUnit, TSecTm(1), utMedia, *QtBs);
    const bool Peak1B = Qt.IsSinglePeak(TmUnit, TSecTm(1), utBlog, *QtBs);
    Qt.GetFqOt(RawFqV, tmuDay, utUndef, *QtBs);
    TQuote::GetSmoothFqOt(FqV, RawFqV, TmUnit, 6*24*3600/TmUnitSecs, 1.3);
    for (int i = 0; i < FqV.Len(); i++) {
      AllMedBlgH.AddDat(FqV[i].Val1).Val1 = FqV[i].Val2;
    }
    Qt.GetFqOt(RawFqV, tmuDay, utMedia, *QtBs);
    TQuote::GetSmoothFqOt(FqV, RawFqV, TmUnit, 6*24*3600/TmUnitSecs, 1.3);
    for (int i = 0; i < FqV.Len(); i++) {
      AllMedBlgH.AddDat(FqV[i].Val1).Val2 = FqV[i].Val2;
    }
    Qt.GetFqOt(RawFqV, tmuDay, utBlog, *QtBs);
    TQuote::GetSmoothFqOt(FqV, RawFqV, TmUnit, 6*24*3600/TmUnitSecs, 1.3);
    for (int i = 0; i < FqV.Len(); i++) {
      AllMedBlgH.AddDat(FqV[i].Val1).Val3 = FqV[i].Val2; }
    // save
    TFltPrV allV, medV, blgV;
    for (int i = 0; i < AllMedBlgH.Len(); i++) {
      allV.Add(TFltPr(i, AllMedBlgH[i].Val1));
      medV.Add(TFltPr(i, AllMedBlgH[i].Val2));
      blgV.Add(TFltPr(i, AllMedBlgH[i].Val3));
    }
    TStr::Fmt("C:%d  #:%d  U:%d  %s", QtBs->GetClust(QtIdV[q]).Len(), Qt.GetFq(), Qt.GetUrls(), Qt.GetStr().CStr()).Save(FOut);
    medV.Save(FOut);
    blgV.Save(FOut);
    NPeak1++;
    //TStr FNm;
    //if (Peak1M && Peak1B) { FNm = TStr::Fmt("qtOt-1peak-%04d", q); NPeak1++; }
    //else { FNm = TStr::Fmt("qtOt-multPeak-%04d", q); }
    //TGnuPlot GP(FNm, TStr::Fmt("C:%d  #:%d  U:%d  %s", QtBs->GetClust(QtIdV[q]).Len(),
    //  Qt.GetFq(), Qt.GetUrls(), Qt.GetStr().CStr()));
    //GP.AddPlot(allV, gpwLinesPoints, "ALL");
    //GP.AddPlot(medV, gpwLinesPoints, "MEDIA");
    //GP.AddPlot(blgV, gpwLinesPoints, "BLOGS");
    //GP.SetXYLabel(TStr::Fmt("TIME [%s]", TTmInfo::GetTmUnitStr(TmUnit).CStr()), "Count (smooth)");
    //GP.SavePng();
    fprintf(F, "%d\t%d\t%d\t\t%s\t%s\t%s\n", QtBs->GetClust(QtIdV[q]).Len(), Qt.GetFq(), Qt.GetUrls(),
      Peak1M?"M1P":"MMP", Peak1B?"B1P":"BMP", Qt.GetStr().CStr());  fflush(F);
  } // */

  // X: cluster quotes fq>10 (use old rules, no merge if words>10)
  // Y: cluster quotes fq>10 and doms*4>urls
  //QtBs->GetClustNet(10, "qt081018w4Mfq10Y")->Save(TFOut("qt081018w4Mfq10Y.ClustNet"));
  /*QtBs->ResetClusters();
  QtBs->ClusterQts(6, 10, "qt081018w4Mfq10-ClustMFq10Y");
  { TFOut FOut("qt081018w4Mfq10-ClustMFq10Y.QtBs"); QtBs->Save(FOut); } // */

  //PClustNet Net = TClustNet::Load(TFIn("qt081018w4Mfq10.ClustNet"));
  //Net = Net->GetSubGraph(5, 0);
  //Net->DrawNet("qt081018w4Mfq10", 50);
  //TGraphViz::Plot(Net, gvlDot, "qt081018w4Mfq10-09.gif");

  //// Save txt clusters
  //TQuoteBs QBs(TZipIn("qt080901w4-1dayClustMFq10.QtBs.rar"));
  //QBs.SaveClusters("clust", false);

  //// Count quote frequency outside text
  //TStrV QuoteStrV;
  //TStr("thanks but no thanks|lipstick on a pig|bridge to nowhere|nation of whiners|palling around|community organizer|long and painful recession|most serious financial crisis since the great depression|just your average hockey mom|average hockey mom|average hockey mom|fundamentals of the economy are strong").SplitOnAllCh('|', QuoteStrV);
  //TQuoteBs::FindQtInTxtQBs("*contentBin*", QuoteStrV)->Save(TFOut("OutOfQuotes-081018.QtBs"));
  //*/
  CatchFull
  printf("\nrun time: %s (%s)\n", ExeTm.GetTmStr(), TSecTm::GetCurTm().GetTmStr().CStr());
  return 0;
}

void PlotsForPaper() {
  //PQuoteBs QtBs1 = TQuoteBs::Load(TZipIn("qt081020w4ClMFq10-oClUt2.QtBs.rar"));
  //PQuoteBs QtBs2 = TQuoteBs::Load(TZipIn("qt0801-081027w4ClMFq10-oClUt2.QtBs.rar"));
  //PQuoteBs QtBs3 = TQuoteBs::Load(TZipIn("qt0801-081029w4ClMFq10-oCl3Ut2.QtBs.rar"));
  //QtBs1->PlotBlogFracOt(tmu12Hour, true, 1000, "test1Ka-12h");
  //QtBs2->PlotMediaVsBlogLag(tmu30Min, true, 1000, "top1kC30min");
  //QtBs2->PlotMediaVsBlogLag(tmu1Hour, true, 1000, "top1kC1h");
  //QtBs->PlotFqDecay(tmu1Hour, true, utUndef, 1000, 100, "clust1k"); // decay-clust1k
  //QtBs3->SaveBigBlogMassQt("bigBlogMass");
}

//// Save QtBs for Flash visualization
void SaveForFlash(const PQuoteBs& QtBs, const TStr Pref, const bool& TakeOnlyRoots, const TUrlTy& CntUrlTy, const TIntV& IncludeCIdV) {
  const int MinWrdLen = 4;
  const int MinQtFq = 100;
  TIntV QtIdV;
  TSecTm MinTm, MaxTm; QtBs->GetMinMaxTm(MinTm, MaxTm);
  QtBs->GetQtIdVByFq(QtIdV, MinWrdLen, MinQtFq, TakeOnlyRoots, "", "", CntUrlTy);
  /*TIntV QtIdV2;
  QtIdV2.AddV(IncludeCIdV);
  for (int i = 0; i < TMath::Mn(1000, QtIdV.Len()); i++) {
    if (! QtIdV2.IsIn(QtIdV[i]) && QtBs->GetQt(QtIdV[i]).GetStr().SearchStr("the day the earth stood still")==-1) {
      QtIdV2.Add(QtIdV[i]); } }
  QtIdV.Swap(QtIdV2);*/
  QtBs->SaveForFlash(QtIdV, Pref+"top30", tmu4Hour, 30, TSecTm(1), TSecTm(TInt::Mx-1), CntUrlTy);
  QtBs->SaveForFlash(QtIdV, Pref+"top50", tmu4Hour, 50, TSecTm(1), TSecTm(TInt::Mx-1), CntUrlTy);
  QtBs->SaveForFlash(QtIdV, Pref+"top100", tmu4Hour, 100, TSecTm(1), TSecTm(TInt::Mx-1), CntUrlTy);
  /*QtBs->SaveForFlash(QtIdV, Pref+"elec30", tmu4Hour, 30, TSecTm(2008,8,24,0,0,0), TSecTm(2008,10,18,0,0,0), CntUrlTy);
  QtBs->SaveForFlash(QtIdV, Pref+"elec50", tmu4Hour, 50, TSecTm(2008,8,24,0,0,0), TSecTm(2008,10,18,0,0,0), CntUrlTy);
  QtBs->SaveForFlash(QtIdV, Pref+"elec100", tmu4Hour, 100, TSecTm(2008,8,24,0,0,0), TSecTm(2008,10,18,0,0,0), CntUrlTy);
  QtBs->GetQtIdVByFq(QtIdV, MinWrdLen, MinQtFq, TakeOnlyRoots, "", CntUrlTy, TSecTm(MaxTm-31*24*3600), MaxTm);
  QtBs->SaveForFlash(QtIdV, Pref+"month50", tmu4Hour, 50, TSecTm(MaxTm-31*24*3600), MaxTm, CntUrlTy);
  QtBs->GetQtIdVByFq(QtIdV, MinWrdLen, MinQtFq, TakeOnlyRoots, "", CntUrlTy, TSecTm(MaxTm-8*24*3600), MaxTm);
  QtBs->SaveForFlash(QtIdV, Pref+"week50", tmu1Hour, 50, TSecTm(MaxTm-8*24*3600), MaxTm, CntUrlTy);
  QtBs->GetQtIdVByFq(QtIdV, MinWrdLen, MinQtFq, TakeOnlyRoots, "econom", CntUrlTy);
  QtBs->SaveForFlash(QtIdV, Pref+"economy50", tmu4Hour, 50, TSecTm(1), TSecTm(TInt::Mx-1), CntUrlTy); // */
}

void SaveQtDomNet(const int& CId, const PQuoteBs& QtBs, const PNGraph& PostGraph, const TStr& OutFNm) {
  TQuote Qt;
  QtBs->GetMergedClustQt(CId, Qt, false);
  PNGraph DomG = TNGraph::New();
  TIntSet UrlSet;
  TStrSet DomH;
  TIntStrH NIdDomH, NIdClrH;
  for (int i = 0; i < Qt.GetTimes(); i++) { UrlSet.AddKey(Qt.GetUrlId(i)); }
  for (int i = 0; i < Qt.GetTimes(); i++) {
    const int url = Qt.GetUrlId(i);
    if (! PostGraph->IsNode(url)) { continue; }
    const int dom = DomH.AddKey(TStrUtil::GetDomNm2(QtBs->GetStr(url)));
    TNGraph::TNodeI NI = PostGraph->GetNI(url);
    for (int o = 0; o < NI.GetOutDeg(); o++) {
      if (! UrlSet.IsKey(NI.GetOutNId(o))) { continue; }
      const int url2 = NI.GetOutNId(o);
      const int dom2 = DomH.AddKey(TStrUtil::GetDomNm2(QtBs->GetStr(url2)));
      if (dom==dom2) { continue; }
      if (! DomG->IsNode(dom)) { DomG->AddNode(dom);  NIdDomH.AddDat(dom, DomH[dom]); }
      if (! DomG->IsNode(dom2)) { DomG->AddNode(dom2);  NIdDomH.AddDat(dom2, DomH[dom2]); }
      DomG->AddEdge(dom, dom2);
      if (QtBs->GetUrlTy(url)==utMedia) { NIdClrH.AddDat(dom, "Blue"); } // media is blue
      else if (! NIdClrH.IsKey(dom)) { NIdClrH.AddDat(dom, QtBs->GetUrlTy(url)==utMedia?"Blue":"Red"); }
      if (QtBs->GetUrlTy(url2)==utMedia) { NIdClrH.AddDat(dom2, "Blue"); } // media is blue
      else if (! NIdClrH.IsKey(dom2)) { NIdClrH.AddDat(dom2, QtBs->GetUrlTy(url2)==utMedia?"Blue":"Red"); }
    }
  }
  //TSnap::PrintInfo(DomG, Qt.GetStr());
  //TGraphViz::Plot<PNGraph>(DomG, gvlNeato, OutFNm+".gif", Qt.GetStr(), NIdDomH);
  TSnap::SavePajek(DomG, OutFNm+".net", NIdClrH, NIdDomH);
  TSnap::SavePajek(TSnap::GetMxWcc(DomG), "wcc-"+OutFNm+".net", NIdClrH, NIdDomH);
  TSnap::DelDegKNodes(DomG, 1, -1);
  TSnap::SavePajek(DomG, "del1-"+OutFNm+".net", NIdClrH, NIdDomH);
  printf("%s :  %s\n", OutFNm.CStr(), Qt.GetStr().CStr());
}

void GetStoryLines() {
  // two quotes are in the same cluster if the appear at same urls
  PQuoteBs QtBs;
  { TZipIn ZipIn("QtBase-qt0808-0902w3mfq5\\qt08080902w3mfq5-clust.QtBs.rar");
  QtBs = TQuoteBs::Load(ZipIn); }
  TIntV CIdV;  QtBs->GetCIdVByFq(CIdV, 100);
  TVec<TPair<TStr, TIntV> > QtUrlV;
  for (int c = 0; c < 1000; c++) {
    TQuote Qt;  QtBs->GetMergedClustQt(CIdV[c], Qt, false);
    TIntV UrlIdV;
    for (int u = 0; u < Qt.GetTimes(); u++) {
      const int url = Qt.GetUrlId(u);
      UrlIdV.Add(url);
    }
    UrlIdV.Sort();
    QtUrlV.Add(TPair<TStr, TIntV>(Qt.GetStr(), UrlIdV));
  }
  for (int q1 = 0; q1 < QtUrlV.Len(); q1++) {
    if (QtUrlV[q1].Val2.Empty()) { continue; }
    printf("%s\n", QtUrlV[q1].Val1.CStr());
    for (int q2 = q1+1; q2 < QtUrlV.Len(); q2++) {
      const int overlap = QtUrlV[q1].Val2.IntrsLen(QtUrlV[q2].Val2);
      if (overlap > 100) {
      //if (overlap > 10 && overlap/(double)TMath::Mx(QtUrlV[q1].Val2.Len(), QtUrlV[q2].Val2.Len()) > 0.1) {
        QtUrlV[q2].Val2.Clr();
        printf("%s\n", QtUrlV[q2].Val1.CStr());
      }
    }
    printf("\n");
  }
  /* // bipartite graph of quotes to urls
  PNGraph CIdUrlG = TNGraph::New();
  for (int c = 0; c < 1000; c++) {
    const int CId = CIdV[c];
    const int CIdNId = Mega(100)+CId;
    TQuote Qt;  QtBs->GetMergedClustQt(CId, Qt, false);
    CIdUrlG->AddNode(CIdNId);
    for (int u = 0; u < Qt.GetTimes(); u++) {
      const int url = Qt.GetUrlId(u);
      if (! CIdUrlG->IsNode(url)) { CIdUrlG->AddNode(url); }
      CIdUrlG->AddEdge(CIdNId, url); // quotes to urls
    }
  }
  TSnap::PrintInfo(CIdUrlG);
  TIntV DelNIdV;
  for (TNGraph::TNodeI NI = CIdUrlG->BegNI(); NI < CIdUrlG->EndNI(); NI++) {
    if (NI.GetInDeg() > 1) { DelNIdV.Add(NI.GetId()); }
  }
  printf("del %d nodes\n", DelNIdV.Len());
  TSnap::DelNodes(CIdUrlG, DelNIdV);
  TSnap::PrintInfo(CIdUrlG);

  TSnap::PlotWccDistr(CIdUrlG, "CidUrlG", "Bipartite Quotes to Urls graph");
  TCnComV CnCom;  TSnap::GetWccs(CIdUrlG, CnCom);
  for (int c = 0; c < CnCom.Len(); c++) {
    printf("%d  %d\n", c, CnCom[c].Len());
    for (int i = 0; i < CnCom[c].Len(); i++) {
      if(CnCom[c][i] >= Mega(100)) {
        const int cid = CnCom[c][i] - Mega(100);
        TQuote Qt;  QtBs->GetMergedClustQt(cid, Qt, false);
        printf("%s\n", Qt.GetStr().CStr());
      }
    }
    printf("\n");
  } // */
}

void BigMain(int argc, char* argv[]) {
  TExeTm ExeTm;
  Env = TEnv(argc, argv, TNotify::StdNotify);
  Env.PrepArgs("QuotesApp");
  const TStr ToDo = Env.GetIfArgPrefixStr("-do:", "", "To do").GetLc();
  if (Env.IsEndOfRun()) {
    printf("To do:\n");
    printf("    MkDataset         : Make memes dataset (extract quotes and save txt)\n");
    printf("    MemesToQtBs       : Load memes dataset and create quote base\n");
    printf("    AddQuoteBs        : Add ContentBin to QuoteBase structure\n");
    printf("    MkTopQtBs         : Build the QuoteBase from top most frequent quotes over time\n");
    printf("    EvalClust         : Evaluate different quote clustering strategies\n");
    printf("    CntUrls           : Count different urls and news sources\n");
    printf("    QtSubSeq          : Count quote subsequences\n");
    printf("    CntNGrams         : Count n-gram frequencies\n");
    printf("    LinkCascades      : Get cascades from links using MkDataset data\n");
    printf("    DoPl9ot:do plots fo rhte paper");
    return;
  }
  // extract quotes
  // extract quotes and links and make them into a single file
  if (ToDo == "mkdataset") {
    const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "files.txt", "Spinn3r input files (one file per line)");
    const TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "Spinn3r-dataset.txt", "Output file");
    const int MinQtWrdLen = Env.GetIfArgPrefixInt("-w:", 3, "Minimum quote word length");
    const TStr UrlFNm = Env.GetIfArgPrefixStr("-u:", "", "Seen url set (THashSet<TMd5Sig>) file name");
    const bool UrlOnlyOnce = Env.GetIfArgPrefixBool("-q:", true, "Only keep unique Urls");
    //// parse directly from Spinn3r
    TStr Spinn3rFNm;
    THashSet<TMd5Sig> SeenUrlSet;
    if (UrlOnlyOnce && ! UrlFNm.Empty()) {  // keep track of already seen urls (so that there are no duplicate urls)
      TFIn FIn(UrlFNm);  SeenUrlSet.Load(FIn);
    }
    FILE *F = fopen(OutFNm.CStr(), "wt");
    TFIn FIn(InFNm);
    int Items=0;
    for (int f=0; FIn.GetNextLn(Spinn3rFNm); f++) {
      TQuoteExtractor QE(Spinn3rFNm.ToTrunc());
      printf("Processing %02d: %s [%s]\n", f+1, Spinn3rFNm.CStr(), TExeTm::GetCurTm());
      fflush(stdout);
      for (int item = 0; QE.Next(); item++) {
        const TMd5Sig PostMd5(QE.PostUrlStr);
        if (QE.QuoteV.Empty() && QE.LinkV.Empty()) { continue; } // no quotes, no links
        if (UrlOnlyOnce) {
          if (SeenUrlSet.IsKey(PostMd5)) { continue; }
          SeenUrlSet.AddKey(PostMd5);
        }
        fprintf(F, "P\t%s\n", QE.PostUrlStr.CStr());
        fprintf(F, "T\t%s\n", QE.PubTm.GetYmdTmStr().CStr());
        for (int q = 0; q < QE.QuoteV.Len(); q++) {
          if (TStrUtil::CountWords(QE.QuoteV[q]) >= MinQtWrdLen) {
            fprintf(F, "Q\t%s\n", QE.QuoteV[q].CStr()); }
        }
        for (int l = 0; l < QE.LinkV.Len(); l++) {
          fprintf(F, "L\t%s\n", QE.LinkV[l].CStr()); }
        fprintf(F, "\n");
        if (item>0 && item % Kilo(100) == 0) {
          QE.DumpStat();  QE.ExeTm.Tick(); }
        Items++;
      }
      printf("file done. Total %d all posts, %d all items\n", SeenUrlSet.Len(), Items);
      fflush(stdout);
    }
    printf("all done. Saving %d post urls\n", SeenUrlSet.Len());  fflush(stdout);
    if (! SeenUrlSet.Empty()) {
      TFOut FOut(OutFNm.GetFMid()+".SeenUrlSet");
      SeenUrlSet.Save(FOut);
    }
    fclose(F);
  }
  // load memes dataset (MkDataset) and create quote base
  else if (ToDo == "memestoqtbs") {
    const TStr InFNmWc = Env.GetIfArgPrefixStr("-i:", "quotes_200*.txt.rar", "Input Memes dataset files");
    const TStr BlackListFNm = Env.GetIfArgPrefixStr("-b:", "quote_blacklist.txt", "Quote black list file");
    const TStr MediaUrlFNm = Env.GetIfArgPrefixStr("-u:", "../../news_media.txt", "Fule with news media urls");
    const TStr FNmPref = Env.GetIfArgPrefixStr("-o:", "qt08080902", "Output quote base");
    const int MinWrdLen = Env.GetIfArgPrefixInt("-w:", 4, "Min quote word length");
    const int MinMemeFq = Env.GetIfArgPrefixInt("-m:", 5, "Min meme frequency");
    THashSet<TChA> FqMemeSet, BlackList;
    TChA Ln;
    for (TFIn FIn(BlackListFNm); FIn.GetNextLn(Ln); ) {
      BlackList.AddKey(Ln);
    }
    printf("Black list: %d\n", BlackList.Len());
    { TStrHash<TInt, TBigStrPool> MemeCntH(Mega(5));
    for (TMemesDataLoader Memes(InFNmWc); Memes.LoadNext(); ) {
      for (int m = 0; m < Memes.MemeV.Len(); m++) {
        MemeCntH.AddDat(Memes.MemeV[m].CStr()) += 1;
      }
    }
    printf("done [%s]\nAll quotes: %d\n", ExeTm.GetStr(), MemeCntH.Len());
    { TFOut FOut(FNmPref+"-MemeFq.strHash");
      MemeCntH.Save(FOut); }
    for (int i = 0; i < MemeCntH.Len(); i++) {
      if (MemeCntH[i] >= 5 && ! BlackList.IsKey(MemeCntH.GetKey(i))) {
        FqMemeSet.AddKey(MemeCntH.GetKey(i)); }
    } }
    printf("Frequent quotes: %d\n", FqMemeSet.Len());
    PQuoteBs QtBs = TQuoteBs::New();
    for (TMemesDataLoader Memes(InFNmWc); Memes.LoadNext(); ) {
      for (int m = 0; m < Memes.MemeV.Len(); m++) { // delete non-frequent memes
        if (! FqMemeSet.IsKey(Memes.MemeV[m])) {
          Memes.MemeV[m].Clr(); }
      }
      QtBs->AddQuote(Memes.MemeV, Memes.LinkV, Memes.PostUrlStr, Memes.PubTm, 3);
    }
    QtBs->SetUrlTy(MediaUrlFNm, utMedia);  // set url type
    { TFOut FOut(TStr::Fmt("%sw%dmfq%d.QtBs", FNmPref.CStr(), MinWrdLen, MinMemeFq));
    QtBs->Save(FOut); }
  }
  // make cluster network
  else if (ToDo == "mkclustnet") {
    const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "QtBase-qt0808-0902w3mfq5\\qt08080902w3mfq5-clust.QtBs.rar", "Input quote base");
    const TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "qt08080902w3mfq5", "Output network/updated QtBs filename");
    PQuoteBs QtBs;
    if (TZipIn::IsZipFNm(InFNm)) { TZipIn ZipIn(InFNm);  QtBs = TQuoteBs::Load(ZipIn); }
    else { TFIn FIn(InFNm);  QtBs = TQuoteBs::Load(FIn); }
    PClustNet Net = TClustNet::GetFromQtBs(QtBs);
    TSnap::PrintInfo(Net);
    { TFOut FOut(OutFNm+".ClustNet"); Net->Save(FOut); }
    // cluster
    TSnap::DelSelfEdges(Net);
    TIntPrV KeeepEdgeV;
    Net->ClustKeepSingleEdge(1, KeeepEdgeV);
    Net->MakeClusters(KeeepEdgeV);
    TVec<TIntV> ClustV;
    Net->GetClusters(ClustV);
    QtBs->CreateClusters(ClustV); // set clusters
    TFOut FOut(OutFNm+"-clust.QtBs");
    QtBs->Save(FOut);
  }
  /////////////////////////////////////////////////////////////////////////////////////////////////////////
  // add to quote base data structure
  else if (ToDo == "addquotebs") {
    const TStr InFNmWc = Env.GetIfArgPrefixStr("-i:", "fullSpinn3r*.txt.gz", "Input fullSpinnr files");
    const TStr QtBsFNm = Env.GetIfArgPrefixStr("-q:", "qt081011.QtBs", "Input quote base");
    const TStr BlackList = Env.GetIfArgPrefixStr("-b:", "quote_blacklist.txt", "Quote black list file");
    const TStr FNmPref = Env.GetIfArgPrefixStr("-o:", "qt081012", "Output quote base");
    const int MinWrdLen = Env.GetIfArgPrefixInt("-w:", 4, "Min quote word length");
    PQuoteBs QtBs;
    if (TZipIn::IsZipFNm(QtBsFNm)) { TZipIn ZipIn(QtBsFNm); QtBs=TQuoteBs::Load(ZipIn); }
    else { TFIn FIn(QtBsFNm); QtBs=TQuoteBs::Load(FIn); }
    TMakeQtBsFromSpinn3r AddQtBs(QtBs, MinWrdLen);
    //if (! BlackList.Empty()) { AddQtBs.AddToQtBlackList(BlackList); }
    AddQtBs.ProcessPosts(InFNmWc);
    printf("done. time: %s. save QuoteBs..\n", TExeTm::GetCurTm());
    { TFOut FOut(FNmPref+".QtBs");
    QtBs->Save(FOut); }
  }
  // make quote base from a set of most frequent quotes
  if (ToDo == "mktopqtbs") {
    const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "qt080824-081006.QtBs.rar", "Input QtBs file");
    const TStr FNmPref = Env.GetIfArgPrefixStr("-o:", "qt080824-080921", "Output file prefix");
    const int TmStepHr = Env.GetIfArgPrefixInt("-s:", 24, "Time step size [hours]");
    const int TmWndSzHr = Env.GetIfArgPrefixInt("-w:", 24, "Time window size [hours]");
    const int MinWrdLen = Env.GetIfArgPrefixInt("-l:", 4, "Minimum quote length in [words]");
    const int FqToDomFac = Env.GetIfArgPrefixInt("-f:", 4, "Take if QuoteFreq < F * QuoteDomains");
    const int MinQtFq = Env.GetIfArgPrefixInt("-fq:", 5, "Minimum quote frequency");
    const int TakePerStep = Env.GetIfArgPrefixInt("-n:", 100, "Take top N quotes per time step");
    TZipIn FIn(InFNm);
    PQuoteBs QtBs = TQuoteBs::Load(FIn);
    TIntV QIdV;// = QtBs->GetTopLongQuotes(TmWndSzHr, TmStepHr, MinWrdLen, MinQtFq, FqToDomFac, TakePerStep);
    Fail;
    { TFOut FOut(FNmPref+TStr(".QtBs"));
    PQuoteBs newQBs = QtBs->GetQuoteBs(QIdV);
    newQBs->Save(FOut); }
  }
  // evaluate the quality of different quote clustering approaches
  else if (ToDo == "evalclust") {
    //PClustNet Net = TClustNet::Load(TFIn("QtBs-0801-1231\\qt08011231w5.ClustNet"));
    PClustNet Net; { TFIn FIn("QtBs-0801-1231\\qt08011231w5ClMFq10.ClustNet"); Net = TClustNet::Load(FIn); }
    //PClustNet Net = TClustNet::Load(TFIn("QtBs\\qt0810-081022w4.ClustNet"));
    TSnap::PrintInfo(Net, "", "", false);
    Net->ClustGreedyTopDown();
    Net->ClustKeepSingleEdge(1);
    Net->ClustKeepSingleEdge(2);
    Net->ClustKeepSingleEdge(3);
    Net->ClustKeepSingleEdge(4);
    Net->ClustGreedyRandom();
  }
  //
  else if (ToDo == "cnturls") {
    int TotFq=0;
    TStr Ln; THashSet<TMd5Sig> UrlSet;
    for (TQuoteLoader QL("content2\\*.contentBin*"); QL.Next(); TotFq++) { UrlSet.AddKey(TMd5Sig(QL.BlogUrlStr)); }
    //for (TFIn FIn("W:\\Data\\Blogs-Spinn3r-full\\sources_aug1-nov1.txt"); FIn.GetNextLn(Ln); TotFq++) { UrlSet.AddKey(TMd5Sig(Ln)); }
    printf("total items: %d\n", TotFq);
    printf("total different sources (blog urls): %d\n", UrlSet.Len());
  }

  //*/
  //QtBs->SaveBigBlogMassQt("bigBlogMass");
  /*QtBs->PlotPopularityCnt(false, utUndef, TInt::Mx, "qtLipstickPig-all");
  QtBs->PlotPopularityCnt(false, utMedia, TInt::Mx, "qtLipstickPig-media");
  QtBs->PlotPopularityCnt(false, utBlog, TInt::Mx, "qtLipstickPig-blog");
  QtBs->PlotPopularityCnt(true, utUndef, TInt::Mx, "clustAll");
  QtBs->PlotPopularityCnt(true, utMedia, TInt::Mx, "clustMed");
  QtBs->PlotPopularityCnt(true, utBlog, TInt::Mx, "clustBlog");
  QtBs->PlotPopularityCnt(false, utUndef, TInt::Mx, "qtAll");
  QtBs->PlotPopularityCnt(false, utMedia, TInt::Mx, "qtMedia");
  QtBs->PlotPopularityCnt(false, utBlog, TInt::Mx, "qtBlog");*/

  //// Blog vs. Media Lag and Decay
  //PQuoteBs QtBs = TQuoteBs::Load(TZipIn("qt081020w4ClMFq10-oClUt2.QtBs.rar"));
  //TStrIntPrV DomCntV;
  //QtBs->GetTopQtDoms(DomCntV, 1000, 10, "", utUndef);
  //QtBs->TopDomsByLag(tmu1Hour, 100, 1000);
  //QtBs->TopDomsByLag(tmu1Hour, 200, 1000);
  //QtBs->TopDomsByLag(tmu1Hour, 500, 1000);
  //QtBs->TopDomsByLag(tmu1Hour, 10000, 100, 30);
  //const int  doms= atoi(argv[1]); // 10000
  //const int  clust= atoi(argv[2]); // 100
  //const int  thresh= atoi(argv[3]); // 30
  //QtBs->TopDomsByLag(tmu1Hour, doms, clust, thresh);
  //QtBs->PlotMediaVsBlogLag(tmu1Hour, true, 100, "top100C1H");
  //QtBs->PlotMediaVsBlogLag(tmu30Min, true, 100, "top100C30min");
  //QtBs->PlotMediaVsBlogLag(tmu15Min, true, 100, "top100C15min");
  //QtBs->PlotMediaVsBlogLag(tmu10Min, true, 100, "top100C10min");
  //QtBs->PlotMediaVsBlogLag(tmu1Hour, true, 1000, "top1kC1H");
  //QtBs->PlotMediaVsBlogLag(tmu30Min, true, 1000, "top1kC30min");
  //QtBs->PlotMediaVsBlogLag(tmu15Min, true, 1000, "top1kC15min");
  //QtBs->PlotMediaVsBlogLag(tmu10Min, true, 1000, "top1kC10min");
  ///tBs->PlotMediaVsBlogLag(tmu1Hour, false, 1000, "top1kQ1med");
  //QtBs->PlotMediaVsBlogLag(tmu2Hour, false, 1000, "top1kQ2med");
  //QtBs->PlotMediaVsBlogLag(tmu4Hour, false, 1000, "top1kQ4med");
  //QtBs->PlotMediaVsBlogLag(tmu1Hour, false, 100, "top100Q1med");
  //QtBs->PlotMediaVsBlogLag(tmu2Hour, false, 100, "top100Q2med");
  //QtBs->PlotMediaVsBlogLag(tmu4Hour, false, 100, "top100Q4med");
  //QtBs->PlotFqDecay(tmuDay, true, utUndef, 100, "clust100nsDay-ALL");
  //QtBs->PlotFqDecay(tmu4Hour, true, utUndef, 500, "clust500ns4h-ALL");
  //QtBs->PlotFqDecay(tmu1Hour, true, utUndef, 100, "clust100ns1h-ALL");
  //QtBs->PlotFqDecay(tmu1Hour, true, utUndef, 500, "clust500ns1h-ALL");
  //QtBs->PlotFqDecay(tmu12Hour, true, utUndef, 100, "clust100ns12h-ALL");
  //QtBs->PlotFqDecay(tmu12Hour, true, utUndef, 500, "clust500ns12h-ALL");
  //QtBs->PlotFqDecay(tmuDay, true, utUndef, 100, "clust100nsDay-ALL");
  //QtBs->PlotFqDecay(tmuDay, true, utUndef, 500, "clust500nsDay-ALL");
  //QtBs->PlotFqDecay(tmuDay, true, utBlog, 100, "clust100nsDay-BLOGS");
  //QtBs->PlotFqDecay(tmu4Hour, true, utBlog, 500, "clust500ns4h-BLOGS");
  //QtBs->PlotFqDecay(tmu1Hour, true, utBlog, 100, "clust100ns1h-BLOGS");
  //QtBs->PlotFqDecay(tmu1Hour, true, utBlog, 500, "clust500ns1h-BLOGS");
  //QtBs->PlotFqDecay(tmu12Hour, true, utBlog, 100, "clust100ns12h-BLOGS");
  //QtBs->PlotFqDecay(tmu12Hour, true, utBlog, 500, "clust500ns12h-BLOGS");
  //QtBs->PlotFqDecay(tmuDay, true, utBlog, 100, "clust100nsDay-BLOGS");
  //QtBs->PlotFqDecay(tmuDay, true, utBlog, 500, "clust500nsDay-BLOGS");
  //QtBs->PlotFqDecay(tmuDay, true, utMedia, 100, "clust100nsDay-MEDIA");
  //QtBs->PlotFqDecay(tmu4Hour, true, utMedia, 500, "clust500ns4h-MEDIA");
  //QtBs->PlotFqDecay(tmu1Hour, true, utMedia, 100, "clust100ns1h-MEDIA");
  //QtBs->PlotFqDecay(tmu1Hour, true, utMedia, 500, "clust500ns1h-MEDIA");
  //QtBs->PlotFqDecay(tmu12Hour, true, utMedia, 100, "clust100ns12h-MEDIA");
  //QtBs->PlotFqDecay(tmu12Hour, true, utMedia, 500, "clust500ns12h-MEDIA");
  //QtBs->PlotFqDecay(tmuDay, true, utMedia, 100, "clust100nsDay-MEDIA");
  //QtBs->PlotFqDecay(tmuDay, true, utMedia, 500, "clust500nsDay-MEDIA");
  //QtBs->PlotEmptyY(tmuDay, true, 100, "clust100-day");
  //QtBs->PlotEmptyY(tmuDay, true, 1000, "clust1k-day");
  //QtBs->PlotEmptyY(tmu4Hour, true, 100, "clust100-4h");
  //QtBs->PlotEmptyY(tmu4Hour, true, 1000, "clust1k-4h");
  //QtBs->PlotEmptyY(tmu1Hour, true, 100, "clust100-1h");
  //QtBs->PlotEmptyY(tmu1Hour, true, 1000, "clust1k-1h");
  //
  //
  //
  // save quotes for flash visualization
  else if (ToDo == "saveflash") {
    const TStr QtBsFNm = Env.GetIfArgPrefixStr("-i:", "qt081006w4-mfq5.QtBs.rar", "Input QtBs file");
    const TStr FNmPref = Env.GetIfArgPrefixStr("-o:", "qt", "Output file prefix");
    const TStr SaveNStr = Env.GetIfArgPrefixStr("-s:", "100", "Save top N (separate by ,)");
    const TStr Keyword = Env.GetIfArgPrefixStr("-k:", "", "Save only quotes containing the word").GetLc();
    const int MinWrdLen = Env.GetIfArgPrefixInt("-l:", 4, "Minimum quote length in [words]");
    const int MinQtFq = Env.GetIfArgPrefixInt("-fq:", 5, "Minimum quote frequency");
    const bool DoPlot = Env.GetIfArgPrefixBool("-p:", false, "Plot freq. over time in GnuPlot");
    const bool DoMerge = Env.GetIfArgPrefixBool("-m:", true, "Merge clusters of quotes");
    PQuoteBs QtBs;
    if (TZipIn::IsZipFNm(QtBsFNm)) { TZipIn ZipIn(QtBsFNm); QtBs=TQuoteBs::Load(ZipIn); }
    else { TFIn FIn(QtBsFNm); QtBs=TQuoteBs::Load(FIn); }
    if (DoMerge) {
      QtBs = QtBs->GetMergeClusters(); }
    TIntV QtV;
    QtBs->GetQtIdVByFq(QtV, MinWrdLen, MinQtFq, false, Keyword);
    TStrV SaveStrV; SaveNStr.SplitOnAllCh(',', SaveStrV);
    for (int s = 0; s < SaveStrV.Len(); s++) {
      printf("save top %d / %d quotes\n", SaveStrV[s].GetInt(), QtV.Len());
      TStr OutFNm = TStr::Fmt("%s%s", FNmPref.CStr(), SaveStrV[s].CStr());
      //QtBs->SaveForFlash(QtV, OutFNm, tmu4Hour, SaveStrV[s].GetInt());
      //if (DoPlot) { QtBs->PlotQtCumFqOverTm(QtV, OutFNm, SaveStrV[s].GetInt()); }
      Fail;
    }
  }
  // quote base sub sequences
  else if (ToDo == "qtsubseq") {
    const TStr FNmPref = Env.GetIfArgPrefixStr("-i:", "qt080824-080921", "Input File prefix");
    const TInt TopN = Env.GetIfArgPrefixInt("-t:", 100, "Take top N queries");
    const bool SaveQtBs = Env.GetIfArgPrefixBool("-s:", true, "Save QtBs file with new SubSequences");
    TFIn FIn(FNmPref+".QtBs");
    TQuoteBs QtBs(FIn);
    TIntV TopQIdV;// = QtBs.GetTopLongQuotes(10, 4, TopN);
    QtBs.GetQtIdVByFq(TopQIdV, 4, 10);
    TopQIdV.Del(TopN, TopQIdV.Len());
    //QtBs.BuildQtSubSeqCounts(TopQIdV, TStr::Fmt("%sMin10Fac4-%d.Quote", FNmPref.CStr(), TopN()));
    //if (SaveQtBs) { TFOut FOut(FNmPref+".QtBs");  QtBs.Save(FOut); }
    Fail;
  }
  // extract cascades (from MsDataset inpit file) and save to TXT format
  else if (ToDo == "linkcascades") {
    const TStr InFNmWc = Env.GetIfArgPrefixStr("-i:", "memes_*.rar", "Input file prefix");
    const bool IsInFNmWc = Env.GetIfArgPrefixBool("-w:", true, "Input is wildcard (else a file with list of input files)");
    const TStr StopList = Env.GetIfArgPrefixStr("-t:", "CELF-Cascades/websiteStopList.txt", "Domain name stoplist (delete domains from post net)");
    const TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "CELF-Cascades/Casc5-0808-0908", "Output file");
    const int MinCascSz = Env.GetIfArgPrefixInt("-c:", 5, "Minimum cascade size");
    const bool SavePostNet = Env.GetIfArgPrefixBool("-s:", true, "Save post network");
    const uint StartTm = TSecTm(2008, 1, 1, 0, 0, 0);
    //// build PostNetwork
    TStrHash<TInt> BlogSet;
    THash<TMd5Sig, TPair<TSecTm, TInt> > UrlTmH;
    PNGraph PostNet;// = TNGraph::New(Mega(1), Mega(100));
    int NPosts=0, NSkipPost=0, NSkipLink=0, NLinks=0;
    /*for (TMemesDataLoader Memes(InFNmWc, IsInFNmWc); Memes.LoadNext(); NPosts++) {
      if (NPosts % Kilo(100) == 0) { printf("."); }
      if (NPosts % Mega(1) == 0) {
        printf("%dm[l:%.0f]", NPosts/Mega(1), float(Memes.LineCnt)); fflush(stdout); }
      const TMd5Sig SrcMd5(Memes.PostUrlStr);
      if (UrlTmH.IsKey(SrcMd5)) {
        TSecTm& Tm = UrlTmH.GetDat(SrcMd5).Val1;
        if (! Tm.IsDef()) { Tm = Memes.PubTm; }
        else { NSkipPost++; continue; }
      } else {
        const TChA BlogNm = TStrUtil::GetWebsiteNm(Memes.PostUrlStr);
        const int BlogId = BlogSet.AddKey(BlogNm);
        UrlTmH.AddDat(SrcMd5, TPair<TSecTm, TInt>(Memes.PubTm, BlogId));
      }
      const int SrcNId = UrlTmH.GetKeyId(SrcMd5);
      const int SrcBlog = UrlTmH.GetDat(SrcMd5).Val2;
      NLinks += Memes.LinkV.Len();
      for (int l = 0; l < Memes.LinkV.Len(); l++) {
        const TMd5Sig DstMd5(Memes.LinkV[l]);
        if (! UrlTmH.IsKey(DstMd5)) {
          const TChA BlogNm = TStrUtil::GetWebsiteNm(Memes.LinkV[l]);
          const int BlogId = BlogSet.AddKey(BlogNm);
          if (BlogId == SrcBlog) { NSkipLink++; continue; } // skip links inside same website
          UrlTmH.AddDat(DstMd5, TPair<TSecTm, TInt>(TSecTm(), BlogId));
          //if (BlogNm!=TStrUtil::GetDomNm2(Memes.LinkV[l])) printf("%s\n       %s\n", Memes.LinkV[l].CStr(), BlogNm.CStr());
        }
        const int DstNId = UrlTmH.GetKeyId(DstMd5);
        const int DstBlog = UrlTmH.GetDat(DstMd5).Val2;
        if (SrcBlog == DstBlog) { NSkipLink++; continue; } // skip links inside same website
        PostNet->AddNode(SrcNId);
        PostNet->AddNode(DstNId);
        PostNet->AddEdge(SrcNId, DstNId);
        IAssert(UrlTmH[SrcNId].Val2 != UrlTmH[DstNId].Val2);
      }
    }
    if (SavePostNet) {
      TFOut FOut(OutFNm+".PostNet");
      PostNet->Save(FOut); BlogSet.Save(FOut); UrlTmH.Save(FOut);
    }
    //// save websites
    { FILE *F = fopen(TStr::Fmt("%s-Websites.txt", OutFNm.CStr()).CStr(), "wt");
    TIntH PostCntH;
    for (int i = 0; i < UrlTmH.Len(); i++) {
      PostCntH.AddDat(UrlTmH[i].Val2) += 1; }
    fprintf(F, "#Id\tTotalPosts\tName\n");
    for (int w = 0; w < BlogSet.Len(); w++) {
      fprintf(F, "%d\t%d\t%s\n", w, PostCntH.GetDat(w)(), BlogSet.GetKey(w)); }
    } //*/
    { TZipIn FIn(OutFNm+".PostNet.rar"); PostNet = TNGraph::Load(FIn); BlogSet.Load(FIn); UrlTmH.Load(FIn); }
    printf("\nTotal websites: %d\n", BlogSet.Len());
    printf("Total posts:    %d\n", NPosts);
    printf("Skip posts:     %d\n", NSkipPost);
    printf("Total urls:     %d\n", UrlTmH.Len());
    printf("Total links:    %d\n", NLinks);
    printf("Skip links:     %d\n\n", NSkipLink);
    TSnap::PrintInfo(PostNet);
    return ;
    //// delete posts from the stop-list blogs
    if (TFile::Exists(StopList)) {
      printf("build website stoplist\n");
      TIntSet StopSet;  TStr Ln;
      int lines=0;
      for (TFIn FIn(StopList); FIn.GetNextLn(Ln); lines++) {
        for (int w = 0; w < BlogSet.Len(); w++) {
          if (strstr(BlogSet.GetKey(w), Ln.CStr()) != 0) {
            StopSet.AddKey(w); }
        }
      }
      printf("Stoplist\n  %d lines --> %d websites\n", lines, StopSet.Len());
      TIntV DelNIdV;
      for (TNGraph::TNodeI NI = PostNet->BegNI(); NI < PostNet->EndNI(); NI++) {
        if (StopSet.IsKey(UrlTmH[NI.GetId()].Val2)) { DelNIdV.Add(NI.GetId()); }
      }
      printf("  deleting %d posts\n", DelNIdV.Len());
      TSnap::DelNodes(PostNet, DelNIdV);
    }
    //// blog-blog-citaton network
    /*{ printf("Build blog network\n");
    //PNGraph BlogNet = PNGraph::New();
    TPt<TNodeEDatNet<TInt, TInt> > BlogNet = TNodeEDatNet<TInt, TInt>::New();
    for (TNGraph::TNodeI NI = PostNet->BegNI(); NI < PostNet->EndNI(); NI++) {
      IAssert(NI.GetId() < UrlTmH.Len());
      const int src = UrlTmH[NI.GetId()].Val2;
      if (NI.GetOutDeg() > 0) { BlogNet->AddNode(src); }
      for (int i = 0; i < NI.GetOutDeg(); i++) {
        IAssert(NI.GetOutNId(i) < UrlTmH.Len());
        const int dst = UrlTmH[NI.GetOutNId(i)].Val2;
        BlogNet->AddNode(dst);
        if (! BlogNet->IsEdge(src, dst)) { 
          BlogNet->AddEdge(src, dst, 1); }
        else {
          BlogNet->GetEDat(src, dst) += 1; }
      }
    } 
    TSnap::PrintInfo(BlogNet);
    FILE *F = fopen(TStr::Fmt("%s-BlogNet2.txt", OutFNm.CStr()).CStr(), "wt");
    fprintf(F, "# Directed blog citation network (edge (U,V) if some post at blog U links to some post at blog V)\n", OutFNm.CStr());
    fprintf(F, "# Nodes: %d Edges: %d\n", BlogNet->GetNodes(), BlogNet->GetEdges());
    fprintf(F, "# From\tTo\tLinks\n");
    for (TNodeEDatNet<TInt, TInt>::TEdgeI ei = BlogNet->BegEI(); ei < BlogNet->EndEI(); ei++) {
      fprintf(F, "%d\t%d\t%d\n", ei.GetSrcNId(), ei.GetDstNId(), ei().Val);
    }
    fclose(F);
    BlogNet->Save(TFOut(OutFNm+".BlogNet2")); }
    */
    //// create cascades
    printf("Create cascades\n");
    FILE *F = fopen(TStr::Fmt("%s-Cascades3.txt", OutFNm.CStr()).CStr(), "wt");
    int Cascade = 0;
    /* // old version -- saves CascadeId NodeId Time
    TIntH SzCntH;
    TBreathFS<PNGraph> BFS(PostNet);
    TIntPrV BIdTmV;
    for (TNGraph::TNodeI NI = PostNet->BegNI(); NI < PostNet->EndNI(); NI++) {
      if (! (NI.GetOutDeg()==0 && NI.GetInDeg()!=0)) { continue; }
      if (! UrlTmH[NI.GetId()].Val1.IsDef()) { continue; }
      TExeTm Tm;
      BFS.DoBfs(NI.GetId(), false, true);
      SzCntH.AddDat(BFS.GetNVisited()) += 1;
      if (BFS.GetNVisited() < MinCascSz) { continue; }
      printf("[%g]", Tm.GetSecs());
      BIdTmV.Clr(false);
      for (int i = BFS.NIdDistH.FFirstKeyId(); BFS.NIdDistH.FNextKeyId(i); ) {
        const int NId = BFS.NIdDistH.GetKey(i);
        if (! UrlTmH[NId].Val1.IsDef()) { printf("X"); continue; }
        const int Tm = int(UrlTmH[NId].Val1.GetAbsSecs());
        const int BId = UrlTmH[NId].Val2;
        BIdTmV.Add(TIntPr(Tm, BId));
      }
      BIdTmV.Sort();  Cascade++;
      printf("[%g]", Tm.GetSecs());
      for (int b = 0; b < BIdTmV.Len(); b++) {
        fprintf(F, "%d\t%d\t%d\n", Cascade, BIdTmV[b].Val2(), BIdTmV[b].Val1());
      }
      printf("[%g]c:%d,s:%d  ", Tm.GetSecs(), Cascade, BIdTmV.Len());
    }*/
    fprintf(F, "# CascadeId\tSrcNId\tDstNId\tTimeDelta\n"); fflush(F);
    PNGraph Tree;
    TIntTrV TmSrcDstV;
    TExeTm Tm;
    TPt<TNodeNet<TSecTm> > CascNet;
    TFOut FOut(OutFNm+"-Cascade3.bin");
    for (TNGraph::TNodeI NI = PostNet->BegNI(); NI < PostNet->EndNI(); NI++) {
      if (! (NI.GetOutDeg()==0 && NI.GetInDeg()!=0)) { continue; }
      if (! UrlTmH[NI.GetId()].Val1.IsDef()) { continue; }
      Tree = TSnap::GetBfsTree(PostNet, NI.GetId(), false, true);
      if (Tree->GetNodes() < MinCascSz) { continue; }
      CascNet = TNodeNet<TSecTm>::New();
      TmSrcDstV.Clr(false);
      for (TNGraph::TEdgeI EI = Tree->BegEI(); EI < Tree->EndEI(); EI++) {
        const int SrcNId = EI.GetSrcNId();
        const int DstNId = EI.GetDstNId();
        if (! UrlTmH[SrcNId].Val1.IsDef()) { printf("XXX"); }
        if (! UrlTmH[DstNId].Val1.IsDef()) { printf("YYY"); }
        const TSecTm SrcTm = UrlTmH[SrcNId].Val1;
        const TSecTm DstTm = UrlTmH[DstNId].Val1;
        const int SrcBlog = UrlTmH[SrcNId].Val2;
        const int DstBlog = UrlTmH[DstNId].Val2;
        const int TmDiff = DstTm-SrcTm;
        TmSrcDstV.Add(TIntTr(TmDiff < 0 ? 0:TmDiff , SrcBlog, DstBlog));
        if (TmDiff  < 0) { printf("x"); }
        CascNet->AddNode(SrcBlog, SrcTm);
        CascNet->AddNode(DstBlog, DstTm);
        CascNet->AddEdge(SrcBlog, DstBlog);
      }
      printf("s");
      CascNet->Save(FOut);
      TmSrcDstV.Sort();
      for (int i = 0; i < TmSrcDstV.Len(); i++) {
        fprintf(F, "%d\t%d\t%d\t%d\n", Cascade, TmSrcDstV[i].Val2(), TmSrcDstV[i].Val3(), TmSrcDstV[i].Val1());
      }
      Cascade++;
      printf("[%d,%.1g]", Tree->GetNodes(), Tm.GetSecs());
      fflush(F); FOut.Flush();
    }
    fclose(F);
  }
  printf("\nrun time: %s (%s)\n", ExeTm.GetTmStr(), TSecTm::GetCurTm().GetTmStr().CStr());
}
